
/** 
 * Reads from a StringBuffer representing a line of code and spits out tokens and
 * token types relevant to the assembler language CSE7000
 * 
 * @author Tobin Chee
 * Change Log:<br>
 * - 04/13/09 - Tobin Chee<br>
 * - Changed the way the tokenizer works completely. Now reads from a StringBuffer 
 * instead of a BufferedReader to simplify END_OF_FILE cases<br><br>
 * - 4/14/09 - Dan Carpenter<br>
 * - Changed type of token parameter in getNextToken to StringBuffer<br>
 * 
 */
public interface Tokenizer {
	
	/** Retrieve the next token.
	 * Reads from sourceLine and produces token and the token's type. The sourceLine is partially consumed by this method, such that:
	 * sourceLine = #sourceLine - token<br><br>
	 * 
	 * 
	 * @param sourceLine Source line to read next token from.
	 * @param token Token to output.
	 * @param type type to output.
	 */
	void getNextToken(StringBuffer sourceLine, StringBuffer token, Assembler.TokenType[] type);
}
